{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c303ff65",
   "metadata": {},
   "outputs": [],
   "source": [
    "def grad_huber_obj(y_true, y_pred):\n",
    "    xi = 1.35 \n",
    "    # Though I do not want to make it hard-coded, lightgbm, behind the scene, evaluates the # of parameters\n",
    "    # of the objective function first, then pass according # of parameters. I tried to use partial to set \n",
    "    # the value of xi. It did not work.\n",
    "    # I refer the readers to the source code to have a better understanding of the issue:\n",
    "    # (https://github.com/microsoft/LightGBM/blob/master/python-package/lightgbm/sklearn.py)\n",
    "    y_true, y_pred = np.array(y_true).flatten(), np.array(y_pred).flatten()\n",
    "    N = len(y_true)\n",
    "    resid = y_true - y_pred\n",
    "    ind_m = np.where(np.abs(resid)<=xi)\n",
    "    ind_u = np.where(resid>xi)\n",
    "    ind_l = np.where(resid< -xi)\n",
    "    grad = np.zeros(N)\n",
    "    try:\n",
    "        grad[ind_m] = (-2*(y_true-y_pred))[ind_m]\n",
    "    except:\n",
    "        pass\n",
    "    try:\n",
    "        grad[ind_u] = np.repeat(2*xi,N)[ind_u]\n",
    "    except:\n",
    "        pass\n",
    "    try:\n",
    "        grad[ind_l] = np.repeat(-2*xi,N)[ind_l]\n",
    "    except:\n",
    "        pass\n",
    "    return grad/N\n",
    "\n",
    "# hessian of huber loss with respect to y_pred\n",
    "def hess_huber_obj(y_true, y_pred):\n",
    "    xi = 1.35\n",
    "    y_true, y_pred = np.array(y_true).flatten(), np.array(y_pred).flatten()\n",
    "    N = len(y_true)\n",
    "    resid = y_true - y_pred\n",
    "    ind_m = np.where(np.abs(resid)<=xi)\n",
    "    ind_u = np.where(resid>xi)\n",
    "    ind_l = np.where(resid< -xi)\n",
    "    hess = np.zeros(N)\n",
    "    try:\n",
    "        hess[ind_m] = np.repeat(2,N)[ind_m]\n",
    "    except:\n",
    "        pass\n",
    "    return hess/N\n",
    "\n",
    "# huber loss for lgbm\n",
    "def huber_obj(y_true, y_pred):\n",
    "    grad = grad_huber_obj(y_true, y_pred)\n",
    "    hess = hess_huber_obj(y_true, y_pred)\n",
    "    return grad, hess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "699a964a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from lightgbm import LGBMRegressor\n",
    "\n",
    "params = {\n",
    "    'objective':[None, huber_obj],\n",
    "    'max_depth':[1,2],\n",
    "    'n_estimators':[10,50,100,200,500,1000],\n",
    "    'random_state':[12308],\n",
    "    'learning_rate':[.01,.1]\n",
    "}\n",
    "LGBM = val_fun(LGBMRegressor,params=params,X_trn=X_trn,y_trn=y_trn,X_vld=X_vld,y_vld=y_vld)\n",
    "evaluate(y_trn, LGBM.predict(X_trn), insample=True) \n",
    "evaluate(y_tst, LGBM.predict(X_tst))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc7a6f63",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "params = {\n",
    "    'n_estimators': [300],\n",
    "    'max_depth': [3, 6],\n",
    "    'max_features': [30, 50, 100],\n",
    "    'random_state': [12308]\n",
    "}\n",
    "RF = val_fun(RandomForestRegressor,params=params,X_trn=X_trn,y_trn=y_trn,X_vld=X_vld,y_vld=y_vld)\n",
    "evaluate(y_trn, RF.predict(X_trn), insample=True) \n",
    "evaluate(y_tst, RF.predict(X_tst))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "fhuan",
   "language": "python",
   "name": "fhuan"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
